BEGIN{
	#/doRegister
	#/user/register
	#/statis/vv
	#/playurl
	
	#gawk -v uniqueFields="pid guid"  -v apiFilter="all" -v pidFilter="a1c0f66d02e2a816" -f LogStatist.awk   access/log1.20111211 access/log2.20111211 access/log3.20111211 access/log4.20111211 
	
	#gawk -v uniqueFields="pid guid"  -v apiFilter="/user/register /doRegister" -v pidFilter="all" -f LogStatist.awk   access/log1.20111211 access/log2.20111211 access/log3.20111211 access/log4.20111211 
	
	#允许中按ctrl+t可以看到gawk运行状态
	#uniqueFields:去重字段，默认0，即不去重。相当于联合唯一索引，多个字段用空格或|分隔，统计uv用"guid pid"
	#注意用空格分隔符作为联合唯一索引的时候，每个字段的内容都必须不为"空"才有效，这里不同于数据库中定义的联合唯一索引。例如 guid="xxx" pid=""这样的数据会被忽略掉
	#用|分隔符只要有1项不为空即为合法数据 例如:guid="xxx" pid=""这样的数据不会被忽略掉
	
	#apiFilter:过滤url，正则匹配(match函数)，默认"all"统计所有，统计多个url用空格分隔，统计注册用户是"/user/register /doRegister" ，
	
	#pidFilter:过滤pid，默认"all"统计所有，多个pid用空格分隔，例如某平台对应3个pid，pidFilter="aaa bbb ccc"
	
	#paramFilter:过滤参数 例如:统计vv  -v paramFilter="type=begin" 
	#!字符开始表示不包含-v paramFilter="!type=begin" 统计每行中不包含type=begin的 
	
	#extraIPUnqiue:无guid的是否用ip地址代替，0不代替，1代替，默认0，即没有guid该行log不会被计入uv
	
	#userAgentFilter:user-agent，默认all，统计所有。
	#unSafeUserAgent:非userAgentFilter指定的user-agent访问量
	
	#apiFilter='/statis/vv\\.?(json|xml|text)?'
	
	#notStatistPid 是否包含不统计的pid(wap,内部调用) notStatistPid=none 统计所有pid,多个pid用空格分隔。
	#127f7a5e3a366bd0 （内部调用用得pid）
	#a8c022bcd2adcbee (wap 调api接口用的pid)
	
	#notShowDetail是否显示明细，默认显示
	
	#notCheckGuidValidate 不检查guid是否合法，默认检查
	#notCheckGuidValidate只对uv数据有效
	
	notCheckGuidValidate=notCheckGuidValidate?notCheckGuidValidate:0;
	gsub(/\042|\047/,"",notCheckGuidValidate);
	
	beginTime=systime();
	
	notShowDetail=notShowDetail?notShowDetail:0;
	
	notStatistPid=notStatistPid?notStatistPid:"a8c022bcd2adcbee 127f7a5e3a366bd0";
	if(notStatistPid!="none"){
		gsub(/\042|\047/,"",notStatistPid);
		split(notStatistPid,notStatistPidArray," ");
	}
	for(i=1;i<=length(notStatistPidArray);i++){
		notStatistPidReg=notStatistPidReg"pid="notStatistPidArray[i]"|";
	}
	notStatistPidReg=substr(notStatistPidReg,1,length(notStatistPidReg)-1);	
	notStatistPidArrayLength=length(notStatistPidArray);
	
	unSafeUserAgent=0;
	
	apiFilter=apiFilter?apiFilter:"all";	
	gsub(/\042|\047/,"",apiFilter);
	if(apiFilter){
		split(apiFilter,apiFilterArray," ");
	}
	for(i=1;i<=length(apiFilterArray);i++){
		apiReg=apiReg""apiFilterArray[i]"|";
	}
	apiReg=substr(apiReg,1,length(apiReg)-1);
	
	
	
	
	uniqueFields=uniqueFields?uniqueFields:0;
	uniqueFieldsArrayLength=0;
	uniqueType=1;
	gsub(/\042|\047/,"",uniqueFields);
	tmpuf=uniqueFields;
	if(index(tmpuf,"|")!=0){
		uniqueType=0;
		gsub("|"," ",tmpuf);
	}else{
		uniqueType=1;
		gsub(" ","-",uniqueFields);
		gsub("-"," ",tmpuf);
	}
	if(tmpuf){
		split(tmpuf,uniqueFieldsArray," ");
		uniqueFieldsArrayLength=length(uniqueFieldsArray);
	}
	
	pidFilter=pidFilter?pidFilter:"all";
	if(pidFilter!="all"){
		gsub(/\042|\047/,"",pidFilter);
		split(pidFilter,pidFilterArray," ");
	}
	pidFilterArrayLength=length(pidFilterArray);
	for(i=1;i<=pidFilterArrayLength;i++){
		pidReg=pidReg"pid="pidFilterArray[i]"|";
	}
	pidReg=substr(pidReg,1,length(pidReg)-1);	
	if(pidFilterArrayLength==1 && pidFilter!="all"){
		notShowDetail=1;
	}
	
	
	userAgentFilter=userAgentFilter?userAgentFilter:"all";
	if(userAgentFilter!="all"){
		gsub(/\042|\047/,"",userAgentFilter);
	}	
	
	paramFilter=paramFilter?paramFilter:"any";
	if(paramFilter!="any"){
		gsub(/\042|\047/,"",paramFilter);
		split(paramFilter,paramFilterArray," ");
	}
	for(i=1;i<=length(paramFilterArray);i++){
		paramReg=paramReg""paramFilterArray[i]"|";
	}
	paramReg=substr(paramReg,1,length(paramReg)-1);	
	
	
	extraIPUnqiue=extraIPUnqiue?1:0;
	
	pvTotal=0;
	
	unSafeUserAgent=0;
	
	
	pvTotalResponseTimes=0;
	uvTotalResponseTimes=0;
	
	pvGreaterThan200ms=0;
	uvGreaterThan200ms=0;
	
	slowTimes=0.2;
	
	time=time?time:"";
		
	pidLength=16;	
	guidLength=32;
	
	bt=systime();
	nr=0;
	
	vidFile=vidFile?vidFile:"";
	if(vidFile){
		cmd="rm -fr "vidFile"*"	
    	system(cmd);
    }
	
		
	
}


function findPid(){
	pid="";
	if(match($0,"(pid=[0-9a-z]{"pidLength"})",tmppida)){
		pid=substr(tmppida[1],5);
	}
	return pid;
}

function findGuid(){
	guid="";
	if(match($0,"(guid=[0-9a-z]{"guidLength"})",tmpguida)){
		guid=substr(tmpguida[1],6);
	}
	return guid;
}

function findField(tmpline,field){
	if(field=="pid"){
		return findPid();
	}
	if(field=="guid"){
		return findGuid();
	}
	tmpindex11=index(tmpline,field"=");
	if(tmpindex11){
		tmpline=substr(tmpline,tmpindex11+length(field"="));
		tmpindex11=index(tmpline,"\"");
		if(tmpindex11){
			tmpline=substr(tmpline,1,tmpindex11-1);
			tmpindex11=index(tmpline,"&");
			if(tmpindex11){
				tmpline=substr(tmpline,1,tmpindex11-1);
			}
		}else{
			tmpline="";
		}	
	}else{
		tmpline="";
	}
	return tmpline;	
}

function findValue(tmpline,start,end){
	if(index(tmpline,start)!=0){
		tmpline=substr(tmpline,index(tmpline,start)+length(start)-1);
		tmpline=substr(tmpline,1,index(tmpline,end)-1);	
	}else{
		tmpline="";
	}
	return tmpline;	
}



#效率较低
function setArray1(array,api,pid,rk,responseTimes,slow){
	if(array[api,pid]){
		split(array[api,pid],tmpArray," ");
		split(tmpArray[1],codeArray,",");
		split(codeArray[1],c1,":");
		c1[2]++;
		codestr="0:"c1[2]",";
		codelen=length(codeArray);
		isFind=0;
		for(i=2;i<=codelen;i++){
			split(codeArray[i],kv,":");
			k=kv[1];
			v=kv[2];
			if(k==rk){
				v++;
				isFind=1;
			}
			codestr=codestr""k":"v","
		}
		if(!isFind){
			codestr=codestr""rk":"1","
		}
		codestr=substr(codestr,1,length(codestr)-1);
		tmpArray[2]+=responseTimes;
		tmpArray[3]+=slow;
		array[api,pid]=codestr" "tmpArray[2]" "tmpArray[3];
	}else{
		array[api,pid]="0:1,"rk":1 "responseTimes" "slow;	
	}
}
	
#效率比较高，逻辑简单
function setArray2(array,api,pid,rk,responseTimes,slow){
	if(array[api,pid]){
		split(array[api,pid],tmpArray," ");
		split(tmpArray[1],codeArray,",");
		split(codeArray[1],c1,":");
		c1[2]++;
		codestr="0:"c1[2]",";
		part1=substr(tmpArray[1],index(tmpArray[1],",")+1);
		first=index(part1,rk":");
		if(first){
			ll=length(rk":");		
			tmp=part1;
			part1=substr(tmp,1,first-1);
			part2=substr(tmp,first+ll);
			idx1=index(part2,",");
			if(idx1){
				v=substr(part2,1,idx1-1);
				v++;
				part3=substr(part2,idx1);
				codestr=codestr""part1""rk":"v""part3;	
			}else{	
				part2++;
				codestr=codestr""part1""rk":"part2;
			}		
		}else{
			codestr=codestr""part1","rk":1";
		}	
		tmpArray[2]+=responseTimes;
		tmpArray[3]+=slow;
		array[api,pid]=codestr" "tmpArray[2]" "tmpArray[3];
	}else{
		array[api,pid]="0:1,"rk":1 "responseTimes" "slow;	
	}	
}

#效率较高,逻辑复杂
function setArray(array,api,pid,rk,responseTimes,slow){
	if(array[api,pid]){
		split(array[api,pid],tmpArray," ");
		split(tmpArray[1],codeArray,",");
		split(codeArray[1],c1,":");
		c1[2]++;
		codestr="0:"c1[2]",";
		part1=substr(tmpArray[1],index(tmpArray[1],",")+1);
		first=index(part1,rk":");
		if(first){
			ll=length(rk":");
			if(first==1){	
				part2=substr(part1,first+ll);		
				idx1=index(part2,",");
				if(idx1){
					v=substr(part2,1,idx1-1);
					v++;
					part3=substr(part2,idx1);
					codestr=codestr""rk":"v""part3;
				}else{
					part2++;
					codestr=codestr""rk":"part2;		
				}	
			}else{
				tmp=part1;
				part1=substr(tmp,1,first-1);
				part2=substr(tmp,first+ll);
				idx1=index(part2,",");
				if(idx1){
					v=substr(part2,1,idx1-1);
					v++;
					part3=substr(part2,idx1);
					codestr=codestr""part1""rk":"v""part3;	
				}else{	
					part2++;
					codestr=codestr""part1""rk":"part2;
				}
			}
		}else{
			codestr=codestr""part1","rk":1";
		}	
		tmpArray[2]+=responseTimes;
		tmpArray[3]+=slow;
		array[api,pid]=codestr" "tmpArray[2]" "tmpArray[3];
	}else{
		array[api,pid]="0:1,"rk":1 "responseTimes" "slow;	
	
	}
	
}

{


if($3!="GET" && $3!="POST"){
	next;
}



if(pidFilter=="all" && notStatistPid !="none"){
	if(match($0,notStatistPidReg)){
		next;
	}
}




isPidFilter=0;
pid=0;
if(pidFilter=="all"){
	isPidFilter=1;
}else{	
	if(match($0,pidReg,pidRegArray)){
		if(RLENGTH==pidLength+4){
			isPidFilter=1;	
			pid=pidRegArray[0];
			pid=substr(pid,5);
		}
	}
}




api="";
isApiFilter=0;
if(isPidFilter){
	if(apiFilter=="all"){
		isApiFilter=1;
		api="all";
	}else{
		tmpurl=$4"?"$5$6;
		gsub(/\042|\047/,"",tmpurl);
		if(match(tmpurl,apiReg,apiRegArray)){
			isApiFilter=1;	
			api=apiRegArray[0];
		}
	}
}


isParamFilter=0;
if(isPidFilter && isApiFilter){
	if(paramFilter=="any"){
		isParamFilter=1;
	}else{
		if(index(paramReg,"!")==1){
			tmppr=paramReg;
			sub("!","",tmppr);
			if(match($0,tmppr)==0){
				isParamFilter=1;
			}
		}else{	
			if(match($0,paramReg)){
				isParamFilter=1;
			}
		}
	}
}


isUserAgentFilter=0;
if(isPidFilter && isApiFilter && isParamFilter){
	if(userAgentFilter=="all"){
		isUserAgentFilter=1;
	}else{
		#isUserAgentFilter=index($0,"\""userAgentFilter);
			
		if(NF>10){
			atmpstr=$0;
			atmpstr1=$0;
			atmpn=gsub(" \"","",atmpstr);
			for(i=0;i<atmpn-1;i++){
				sub(" \"","",atmpstr1);
			}
			tmpua=substr(atmpstr1,index(atmpstr1," \"")+2);
			tmpua=substr(tmpua,1,length(tmpua)-1);
		}else{
			tmpua=$10;
		}
		isUserAgentFilter=index(tmpua,userAgentFilter);
	
		if(!isUserAgentFilter){
			unSafeUserAgent++;
		}
	}	
}




if(isUserAgentFilter && isApiFilter && isPidFilter && isParamFilter){
	rk=$7;
	responseTimes=$9;
	if(NF>10 && !(length(rk)==3 && rk>=200 && rk<1000)){
		atmpstr=$0;
		atmpstr1=$0;
		atmpn=gsub("\" ","",atmpstr);
		for(i=0;i<atmpn-1;i++){
			sub("\" ","",atmpstr1);
		}
		rk=substr(atmpstr1,index(atmpstr1,"\" ")+2,3);
		responseTimes=substr(atmpstr1,index(atmpstr1,"\" ")+2);
		split(responseTimes,tmpresp," ");
		responseTimes=tmpresp[3];
	}
	
	
	
	setArray(pvResults,api,"",rk,responseTimes,responseTimes>slowTimes?1:0);
	
	if(!pid){
		pid=findPid();
		if(pid==""){
			pid="nopid";
		}
	}
	if(!notShowDetail){
		setArray(pvResults,api,pid,rk,responseTimes,responseTimes>slowTimes?1:0);
	}
	
	if(vidFile){
		tmpvid=findField($0,"vid");
		if(tmpvid){
			if(!tmpvidunique[tmpvid]){
				tmpvidunique[tmpvid]=1;
			}else{
				tmpvidunique[tmpvid]++;
			}
		}
	}
	
	
	
	
	
	if(uniqueFields){
		lineStr=$0;
		key="";	
		for(i=1;i<=uniqueFieldsArrayLength;i++){
			if(uniqueFieldsArray[i]=="pid"){
				tmpline=pid;
			}else{
				tmpline=findField(lineStr,uniqueFieldsArray[i]);
			}
			
			if(uniqueType==0){
				if(tmpline==""){
					tmpline="no"uniqueFieldsArray[i];
				}
			}	
			
			
			if(tmpline!=""){
				key=key"&"tmpline;		
			}else{
				if(uniqueFieldsArray[i]=="guid" && extraIPUnqiue){
					key=key"&"$1;
				}else{
					key="";
					break;
				}
			}
		}
		if(length(key)>1){
			key=substr(key,2);
		}else{
			key="";
		}	
		if(key!=""){
			key=api"&"key;
			value=1;
			if(key in uvArray){
				;
			}else{
				uvArray[key]=value;
				setArray(uvResults,api,"",rk,responseTimes,responseTimes>slowTimes?1:0);
				if(!notShowDetail){
					setArray(uvResults,api,pid,rk,responseTimes,responseTimes>slowTimes?1:0);
				}				
			}	
		}
	}	

#nr++;	
#if(nr%(10*10000)==0){
#	print systime()-bt;
#	bt=systime();
#}
	
}

}
END{

	

	gsub(" ","|",apiFilter);
	gsub(" ","|",pidFilter);
	gsub(" ","|",notStatistPid);
	
	if(extraIPUnqiue){
		extraIPUnqiue=1;
	}else{
		extraIPUnqiue=0;
	}
	
	if(!uniqueFields){
		uniqueFields="none";
	}
	
	
	strPv=getJson(pvResults,pidFilter);
	strUv=getJson(uvResults,pidFilter);
	resutls="\"pvResults\":"strPv",";
	resutls=resutls"\"uvResults\":"strUv;
	
	
	gsub(/\\/,"\\\\\\",apiFilter);
	
	
	endTime=systime();
	strJson="{\"time\":\""time"\",";
	strJson= strJson"\"apiFilter\":\""apiFilter"\",";
	strJson=strJson"\"paramFilter\":\""paramFilter"\",";
	strJson=strJson"\"pidFilter\":\""pidFilter"\",";
	strJson=strJson"\"notStatistPid\":\""notStatistPid"\",";
	strJson=strJson"\"uniqueFields\":\""uniqueFields"\",\"extraIPUnqiue\":"extraIPUnqiue",";
	strJson=strJson"\"notCheckGuidValidate\":"notCheckGuidValidate",";
	strJson=strJson"\"userAgentFilter\":\""userAgentFilter"\",";
	strJson=strJson"\"unSafeUserAgent\":"unSafeUserAgent",";
	strJson=strJson"\"notShowDetail\":"notShowDetail",";
	strJson=strJson"\"beginTime\":\""strftime( "%Y-%m-%d %T", beginTime)"\",";
	strJson=strJson"\"endTime\":\""strftime( "%Y-%m-%d %T",endTime)"\",";
	strJson=strJson"\"runTime\":"(endTime-beginTime)",";
	strJson=strJson""resutls"}";
	
	print strJson;
	
	if(vidFile){
		tmpTotal=0;
		for(tmpVid in tmpvidunique){
			tmpTotal=tmpTotal+tmpvidunique[tmpVid];
		}
		print "total_vv= "tmpTotal" vid_num = "length(tmpvidunique) >> vidFile;
		print "total_vv = "tmpTotal" vid_num = "length(tmpvidunique)" <br/>" >> vidFile".html";
		for(tmpVid in tmpvidunique){	
			print tmpVid" "tmpvidunique[tmpVid] >> vidFile;
			print "<a href=\"http://v.youku.com/v_show/id_"tmpVid".html\">"tmpVid"</a> "tmpvidunique[tmpVid]" <br/>" >> vidFile".html";
		}
		
		cmd="sort -n -s -t ' ' -k 2 -r -o "vidFile" "vidFile
    	system(cmd);
    	
    	cmd="sort -n -s -t ' ' -k 3 -r -o "vidFile".html "vidFile".html"
    	system(cmd);
	}
	
	
}

function getJson(array,pidFilter){
	if(!length(array)){
		return "[]";
	}

	strResults="";
	slen=asorti(array,tmpA);
	tapi="";
	tpid="";
	json="";
	resp="";
	
	jt="";
	for(i=1;i<=slen;i++){
		
		split(tmpA[i],sp,SUBSEP); 
		api=sp[1];
		pid=sp[2];
		
		gsub(/\\/,"\\\\\\",api);
		
		if(pid==""){
			json="\"api\":\""api"\",";
			json=json"\"pid\":\""pidFilter"\",";
		}else{
			json="\"api\":\""api"\",";
			json=json"\"pid\":\""pid"\",";
		}
		line=array[tmpA[i]];
		split(line,tmpB," ");
		split(tmpB[1],tmpC,",")
		codestr="";
		success=0;
		total=0;
		responsetimes=tmpB[2];
		slow=tmpB[3];
		for(j=1;j<=length(tmpC);j++){
			split(tmpC[j],tmpD,":");
			code=tmpD[1];
			num=tmpD[2];
			
			if(code==200||code==304){
			 	success+=num;
			 }
			
			if(code==0){
				total=num;
			}else{
				codestr=codestr"\"code"code"\":{\"num\":"num",\"per\":\""sprintf("%.5f",num/total*100)"%\"},";
			}
			
		}
		
		jt="\"total\":"total",";
		jt=jt"\"success\":{\"num\":"success",\"per\":\""sprintf("%.5f",success/total*100)"%\"},";
		json=json""jt""codestr;
		json=json"\"averageResponseTimes\":"sprintf("%.5f",responsetimes/total)",";
		json=json"\"slowRequest\":{\"num\":"slow",\"per\":\""sprintf("%.5f",slow/total*100)"%\"},";
		
		json=substr(json,1,length(json)-1);
		
		json="{"json"}";
		resp=resp""json",";
	}
	resp=substr(resp,1,length(resp)-1);
	
	return "["resp"]";
	
	

}